# -*- coding: utf-8 -*-
from src.com.bsjs.util.base_util import BaserUtil
from src.com.bsjs.util.http_util import HttpClient
from src.com.bsjs.util.redis_util import RedisUtil
from src.com.bsjs.util.sql_util import MySQLUtils


class Spider:
    def __init__(self):
        self.http_client = HttpClient()
        self.redis_client=RedisUtil()
        self.base_client=BaserUtil()
        self.mysql_client=MySQLUtils()
        self.headers={
            "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36"
        }

    def obtain_param(self):
        url='https://www.singlewindow.cn/access/ui/SW-SITE-FRONT/Param001'
        data={"Head":{},"Data":"{}"}
        res=self.redis_client.get_value(url)
        if res==None or len(res)==0:
            res=self.http_client.post(data=data,url=url,fast_falg=True)
            self.redis_client.set_value(url,res)
        res=self.base_client.json2obj(res)
        selected_columns=['pqcode','pqname']
        params=[{key: param[key] for key in selected_columns} for param in res['data']]
        return params

    def obtain_data(self, param,try_time):
        url = 'https://www.singlewindow.cn/access/ui/SW-SITE-FRONT/Param002'
        page=self.obtain_data_page(param,try_time)
        datas=[]
        for cur_page in range(1,page+1):
            redis_key=param['pqcode'] + '_' + str(cur_page)
            data = {"Head": {},"Data": "{\"pageNo\":%s,\"rowsPerPage\":10,\"paramName\":\"%s\",\"filterField\":null,\"filterValue\":null}" %(
                            cur_page,param['pqcode'])}
            res = self.redis_client.get_value(redis_key)
            if res == None or len(res) == 0:
                res = self.http_client.post(data=data, url=url,headers=self.headers, fast_falg=True)
                self.redis_client.set_value(redis_key, res)
            res_obj = self.base_client.json2obj(res)

            for i in range(try_time):
                if res_obj['status']!='success':
                    self.base_client.sleep(2)
                    res = self.http_client.post(data=data, url=url,headers=self.headers, fast_falg=False)
                    self.redis_client.set_value(redis_key, res)
                    res_obj = self.base_client.json2obj(res)
                else:
                    break
            print(f"2级目录---》{param['pqname']}当前页码为{cur_page},总页码为{page}当前完成进度为{(((cur_page*100) / page))}%")
            if res_obj['status'] == 'success':
                datas.append(res_obj)
        return datas

    def obtain_data_page(self, param,try_time):
        redis_key=param['pqcode']+'_'+str(1)
        url = 'https://www.singlewindow.cn/access/ui/SW-SITE-FRONT/Param002'
        data = {"Head": {},"Data": "{\"pageNo\":1,\"rowsPerPage\":10,\"paramName\":\"%s\",\"filterField\":null,\"filterValue\":null}" %param['pqcode']}
        res = self.http_client.post(data=data, url=url, fast_falg=True)
        self.redis_client.set_value(redis_key, res)

        for i in range(try_time):
            res_obj=self.base_client.json2obj(res)
            if res_obj['status'] != 'success':
                self.base_client.sleep(2)
                res = self.http_client.post(data=data, url=url, headers=self.headers, fast_falg=False)
                self.redis_client.set_value(redis_key, res)
            else:
                break
        return self.base_client.json2obj(res)['data']['pageSumCount']

    def transfy_dbdata(self, datas,param):
        res=[]
        for data in datas:
            obj={}
            obj['type']=param['pqcode']
            obj['type_zh']=param['pqname']


            selected_columns = ['coldisplay', 'colname']
            head = [{key: d[key] for key in selected_columns} for d in data['data']['head']]
            obj['field']=",".join([ item['coldisplay'] for item in head])
            obj['field1'] = ",".join([ item['colname'] for item in head])
            obj['field_dict'] = self.base_client.obj2json(head)
            obj['page']=data['data']['pageNo']


            db_datas=data['data']['data']
            for db_data in db_datas:
                tmp={}
                tmp.update(obj)
                tmp['field_data']=self.escape_single_quotes(self.base_client.urlencode(self.base_client.obj2json(db_data)))
                res.append(tmp)
        return res

    def escape_single_quotes(self,input_string):
        """
        将输入字符串中的单引号转义为 \'
        :param input_string: 输入的字符串
        :return: 转义后的字符串
        """
        return input_string.replace("'", "\\'")

    def save_data(self, data, param):
        keys=['type', 'type_zh', 'field', 'field1', 'field_dict', 'field_data', 'page']
        db_data = self.transfy_dbdata(data, param)
        if len(db_data) == 0:
            return
        group_list=self.base_client.group_list_by_count(lst=db_data,group_count=10_000)
        for lst in group_list:
            if lst==None or len(lst)==0:
                continue
            self.mysql_client.insertBatch(table='param_data',
                                          keys=keys, data=lst)

    def deal_param(self, param,params):
        count = self.mysql_client.retrieve_count(table='param_data', condition=f"type ='{param['pqcode']}' ")
        if count > 0:
            return
        print(
            f"1级目录---》{param['pqname']}当前页码为：{(params.index(param) + 1)}，总页码为{len(params)}当前完成进度为{((params.index(param) + 1) * 100 / len(params))}%")
        data = self.obtain_data(param, 3)
        self.save_data(data, param)


def main():
    this = Spider()
    params = this.obtain_param()
    for param in params:
        this.deal_param(param,params)






if __name__ == '__main__':
    main()